/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package taganalysistoolsclient;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.tidy.Tidy;

/**
 *
 * @author lendle
 */
public class TestResources {
    public static void main(String [] args) throws Exception{
        URL [] urls=extractWebSitesFromCategory(new URL("http://www.dmoz.org/Computers/Supercomputing/Cray"));
        for(URL url : urls){
            System.out.println(url);
        }
    }
    
    private static URL[] extractWebSitesFromCategory(URL categoryURL) throws Exception{
        List<URL> urls=new ArrayList<URL>();

        BufferedReader input=new BufferedReader(new InputStreamReader(categoryURL.openStream(), "utf-8"));
        try{
            Document doc=new Tidy().parseDOM(input, null);
            //Document doc=db.parse(new InputSource(input));
            NodeList nodeList=doc.getDocumentElement().getElementsByTagName("ul");
            for(int i=0; i<nodeList.getLength(); i++){
                Node node=nodeList.item(i);
                if(node.getNodeType()==Node.ELEMENT_NODE){
                    Element ulElement=(Element) node;
                    if("directory-url".equals(ulElement.getAttribute("class"))){
                        NodeList aNodeList=ulElement.getElementsByTagName("a");
                        for(int j=0; j<aNodeList.getLength(); j++){
                            Element aElement=(Element) aNodeList.item(j);
                            String url=aElement.getAttribute("href");
                            System.out.println(url);
                            urls.add(new URL(url));
                        }
                    }
                }
            }
        }finally{
            input.close();
        }
        return urls.toArray(new URL[0]);
    }
}
